package com.example.demo.service.impl;

import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.example.demo.dao.*;
import com.example.demo.model.Book;

import com.example.demo.service.CrawlerBookMessage;
import com.example.demo.util.HttpUtils;
import org.apache.catalina.valves.CrawlerSessionManagerValve;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;


/**
 * 功能描述：
 * CrawlerBookMessage的实现类，爬取源站是云中书库
 *
 * @Author: Zhanghf
 * @Date: 2021/12/11 16:39
 */
@Service
public class CrawlerMessageYunxs implements CrawlerBookMessage {
    @Autowired
    private HttpUtils httpUtils;
    @Autowired
    private BatchBookServiceImpl batchBookService;
    @Autowired
    private BookMapper bookMapper;

    @Override
    public void crawlermessage() {
        String homeurl = "http://www.yunxs.com/shuku.html";
        String content = httpUtils.doGetHtml(homeurl);
        Document document = Jsoup.parse(content);
        String last = document.getElementsByClass("page_num").
                get(0).getElementsByTag("a").last().attr("href");
        String[] s = last.split("_");
        String[] split = s[2].split("\\.");             //.在正则表达式里作特殊字符，匹配 . 要使用 \. ，但\在JAVA中也是特殊字符，要匹配\，要使用\\
        //split[0] 书库页数
        ArrayList<Book> books = new ArrayList<>();
        List<Object> urls = bookMapper.selectObjs(new QueryWrapper<Book>().select("url"));
        for (int x = 1; x <= Integer.parseInt(split[0]); x++) {
            content = httpUtils.doGetHtml("http://www.yunxs.com/shuku_17801_" + x + ".html");
            document = Jsoup.parse(content);
            Element ul = document.getElementsByClass("ul_m_list").get(0);
            Elements ul_m_list = ul.getElementsByTag("li");
            for (Element li : ul_m_list) {
                String href = li.getElementsByClass("title").get(0).getElementsByTag("a").attr("href");
                //已爬取则跳过
//每爬一个都查一次数据库
//                if (bookMapper.selectOne(new QueryWrapper<Book>().eq("url",homeurl+href))!=null)
//                {
//                    continue;
//                }
//                优化
                if (urls.contains(homeurl + href)) {
                    continue;
                }

                String type = li.getElementsByClass("c").get(0).getElementsByTag("a").text();
                String title = li.getElementsByClass("title").get(0).getElementsByTag("a").attr("title");
                String author = li.getElementsByClass("author").get(0).text();
                String state = li.getElementsByClass("abover").get(0).text();
                Integer wordnumber = Integer.valueOf(li.getElementsByClass("words").get(0).text());
//                源站source设为1，表示云中
                Book book = new Book(title, type, author, 0, 0, 1, "http://www.yunxs.com" + href, state, wordnumber);
                books.add(book);
                // bookMapper.insert(book);
            }
        }
        batchBookService.saveBatch(books, books.size());
        System.out.println("爬取结束");
    }
}
